# -*- coding: utf-8 -*-
"""
数据去重模块 - 任务1.3.3
基于优化后的数据指纹算法实现数据去重
"""

import hashlib
import logging
from typing import Dict, Any, List, Set
from datetime import datetime


class DataDeduplicator:
    """数据去重器"""
    
    def __init__(self):
        self.logger = logging.getLogger(__name__)
        self.seen_fingerprints: Set[str] = set()
    
    def deduplicate_data(self, data_list: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
        """对数据列表进行去重"""
        if not data_list:
            return []
        
        deduplicated_data = []
        duplicate_count = 0
        
        for i, data in enumerate(data_list):
            try:
                fingerprint = self.generate_fingerprint(data)
                
                if fingerprint not in self.seen_fingerprints:
                    self.seen_fingerprints.add(fingerprint)
                    data['data_fingerprint'] = fingerprint
                    deduplicated_data.append(data)
                    self.logger.debug(f"第 {i+1} 条数据通过去重检查")
                else:
                    duplicate_count += 1
                    self.logger.debug(f"第 {i+1} 条数据为重复数据，已跳过")
            
            except Exception as e:
                self.logger.error(f"处理第 {i+1} 条数据时出错: {e}")
                continue
        
        self.logger.info(f"去重完成，输入 {len(data_list)} 条，输出 {len(deduplicated_data)} 条，重复 {duplicate_count} 条")
        return deduplicated_data
    
    def generate_fingerprint(self, data: Dict[str, Any]) -> str:
        """生成数据指纹"""
        try:
            # 提取关键字段用于生成指纹
            key_fields = self._extract_key_fields(data)
            
            # 标准化关键字段
            normalized_fields = self._normalize_fields(key_fields)
            
            # 生成指纹字符串
            fingerprint_string = self._create_fingerprint_string(normalized_fields)
            
            # 生成MD5哈希
            fingerprint = hashlib.md5(fingerprint_string.encode('utf-8')).hexdigest()
            
            self.logger.debug(f"生成指纹: {fingerprint[:8]}... <- {fingerprint_string[:50]}...")
            return fingerprint
        
        except Exception as e:
            self.logger.error(f"生成数据指纹失败: {e}")
            # 返回一个基于时间戳的唯一指纹
            return hashlib.md5(str(datetime.now().timestamp()).encode('utf-8')).hexdigest()
    
    def _extract_key_fields(self, data: Dict[str, Any]) -> Dict[str, str]:
        """提取用于生成指纹的关键字段"""
        key_fields = {}
        
        # 联系人信息（高权重）
        contact_person = data.get('contact_person', '').strip()
        if contact_person:
            key_fields['contact_person'] = contact_person
        
        # 联系方式（高权重）
        contact_info = data.get('contact_info', '').strip()
        if contact_info:
            key_fields['contact_info'] = contact_info
        
        # 标题（中权重）
        title = data.get('title', '').strip()
        if title:
            # 提取标题中的关键信息，去除常见的修饰词
            cleaned_title = self._clean_title_for_fingerprint(title)
            if cleaned_title:
                key_fields['title'] = cleaned_title
        
        # 业态信息（中权重）
        business_type = data.get('business_type', '').strip()
        if business_type:
            key_fields['business_type'] = business_type
        
        # 区域信息（中权重）
        area_info = data.get('area_info', '').strip()
        if area_info:
            key_fields['area_info'] = area_info
        
        # 建筑面积（低权重）
        building_area = data.get('building_area', '').strip()
        if building_area:
            key_fields['building_area'] = building_area
        
        # 租金信息（低权重）
        acceptable_rent = data.get('acceptable_rent', '').strip()
        if acceptable_rent and acceptable_rent != '面议':
            key_fields['acceptable_rent'] = acceptable_rent
        
        return key_fields
    
    def _normalize_fields(self, fields: Dict[str, str]) -> Dict[str, str]:
        """标准化字段值"""
        normalized = {}
        
        for key, value in fields.items():
            if not value:
                continue
            
            # 转换为小写
            normalized_value = value.lower()
            
            # 移除空白字符
            normalized_value = ''.join(normalized_value.split())
            
            # 特殊处理不同字段
            if key == 'contact_info':
                # 联系方式只保留数字
                normalized_value = ''.join(filter(str.isdigit, normalized_value))
            elif key == 'building_area':
                # 面积信息只保留数字和连字符
                normalized_value = ''.join(c for c in normalized_value if c.isdigit() or c in '-~–—')
            elif key == 'acceptable_rent':
                # 租金信息只保留数字和连字符
                normalized_value = ''.join(c for c in normalized_value if c.isdigit() or c in '-~–—')
            
            if normalized_value:
                normalized[key] = normalized_value
        
        return normalized
    
    def _clean_title_for_fingerprint(self, title: str) -> str:
        """清理标题用于指纹生成"""
        if not title:
            return ""
        
        # 移除常见的修饰词和无关信息
        noise_words = [
            '急', '急租', '急找', '寻找', '求租', '找', '需要', '想要',
            '优质', '精装', '豪华', '高档', '便宜', '实惠', '合适',
            '地段好', '位置佳', '交通便利', '配套齐全',
            '先生', '女士', '小姐', '老板', '经理'
        ]
        
        cleaned_title = title
        for word in noise_words:
            cleaned_title = cleaned_title.replace(word, '')
        
        # 移除多余空白
        cleaned_title = ' '.join(cleaned_title.split())
        
        return cleaned_title
    
    def _create_fingerprint_string(self, normalized_fields: Dict[str, str]) -> str:
        """创建用于生成指纹的字符串"""
        # 按字段重要性排序
        field_priority = [
            'contact_info',      # 联系方式最重要
            'contact_person',    # 联系人次之
            'business_type',     # 业态信息
            'area_info',         # 区域信息
            'title',             # 标题
            'building_area',     # 建筑面积
            'acceptable_rent'    # 租金信息
        ]
        
        fingerprint_parts = []
        
        for field in field_priority:
            if field in normalized_fields:
                fingerprint_parts.append(f"{field}:{normalized_fields[field]}")
        
        # 如果没有足够的关键字段，使用所有可用字段
        if len(fingerprint_parts) < 2:
            for field, value in normalized_fields.items():
                if field not in field_priority:
                    fingerprint_parts.append(f"{field}:{value}")
        
        return "|".join(fingerprint_parts)
    
    def reset_seen_fingerprints(self):
        """重置已见过的指纹集合"""
        self.seen_fingerprints.clear()
        self.logger.info("已重置去重缓存")
    
    def get_duplicate_stats(self) -> Dict[str, int]:
        """获取去重统计信息"""
        return {
            'total_fingerprints': len(self.seen_fingerprints)
        }
    
    def is_duplicate(self, data: Dict[str, Any]) -> bool:
        """检查单条数据是否重复"""
        fingerprint = self.generate_fingerprint(data)
        return fingerprint in self.seen_fingerprints
    
    def add_to_seen(self, data: Dict[str, Any]) -> str:
        """将数据添加到已见集合中"""
        fingerprint = self.generate_fingerprint(data)
        self.seen_fingerprints.add(fingerprint)
        return fingerprint
